#import operator
import sys


def dealWithQueryLevelFeature(inputFileName1):
    print "dealWithQueryLevelFeature*******************"
    return_query_Dict = {}
    inputFileHandler1 = open(inputFileName1, "r")
    #keyWordsDict = {}
    
    for currentLine in inputFileHandler1.readlines():
        currentLineElements = currentLine.split("\t")
        queryNumber = currentLineElements[0].strip()
        adhocTopicNumber = currentLineElements[2].strip()
        relevenceScore = currentLineElements[4].strip()
        queryInfoString = queryNumber + " " + adhocTopicNumber + " " + relevenceScore + " "
        
        queryContent = currentLineElements[1].strip()
        warcTrecID = currentLineElements[3].strip()
        
        returnTurple = (queryContent, warcTrecID)
        if returnTurple not in return_query_Dict:
            return_query_Dict[returnTurple] = queryInfoString
        else:
            print "Error1"
            print returnTurple
            sys.exit(1)
            
    
    return return_query_Dict
    #The key is the thing we hunt for.
      
    '''   
        for currentTerm in queryContentElement:
            key = currentTerm + "_" + warcTrecID
            if key not in keyWordsDict:
                keyWordsDict[key] = 1
            else:
                keyWordsDict[key] += 1
    
    keyWordsDict
    sorted_keyWordsDict = sorted(keyWordsDict.iteritems(), key=operator.itemgetter(0))
    
    print len(keyWordsDict)
    print type(sorted_keyWordsDict)
    
    
    for currentElement in sorted_keyWordsDict:
        (queryTerm_warcID,freq) = currentElement
        print queryTerm_warcID
        #print "freq:",freq
    '''
    inputFileHandler1.close()

def dealWithDocumentLevelFeature( dict0, queryDict,inputFileName2 ):
    print "dealWithDocumentLevelFeature*******************"
    #print len(queryDict)
    #print queryDict
    
    documentDict = {}
    NOTRecordedCounter = 0
    inputFileHandler2 = open(inputFileName2, "r")
    for currentLine in inputFileHandler2.readlines():
        currentLineElements = currentLine.strip().split(" ")
        if len(currentLineElements) == 9:
            
            warcTrecID = currentLineElements[0]
            url = currentLineElements[1]
            hamProbability = currentLineElements[3]
            isPopular = currentLineElements[4]    
            urlLength = currentLineElements[5]
            documentSizeInBytes = currentLineElements[6]
            numberOfWords = currentLineElements[7]
            numberOfDistantWords = currentLineElements[8]
            documentInfoString = url + " " + hamProbability + " " + isPopular + " " + urlLength + " " + documentSizeInBytes + " " +  numberOfWords + " " + numberOfDistantWords + " "; 
            
            docID = currentLineElements[2]
            if (warcTrecID,docID) not in documentDict:
                documentDict[ (warcTrecID,docID) ] = documentInfoString
            else:
                print "Error1"
                sys.exit(1)
        else:
            NOTRecordedCounter += 1
                
    inputFileHandler2.close()
    print "# of documents NOT recorded:",NOTRecordedCounter
    #print len(documentDict)
    #print documentDict
    
    #Let's construct a query-document dict
    return_query_document_Dict = {}
    
    for currentElement in queryDict:
        (queryContent,warcTrecID) = currentElement
        
        for eachWARCIDDocIDElement in documentDict:
            (currentWARCID,currentDocID) = eachWARCIDDocIDElement
            
            if warcTrecID == currentWARCID:
                newTurple = (queryContent,currentDocID)
                
                if newTurple not in return_query_document_Dict:
                    return_query_document_Dict[newTurple] = queryDict[currentElement] + documentDict[ (warcTrecID,currentDocID) ]
                else:
                    print "Error2"
                    print newTurple
                    sys.exit(1)

    #print len(return_query_document_Dict)
    #print return_query_document_Dict
    return return_query_document_Dict

def dealWithTermLevelFeature( query_document_Dict , inputFileName3):
    print "dealWithTermLevelFeature*******************"
    #print len(query_document_Dict)
    #print query_document_Dict
    
    
    
    termDict = {}
    
    
    inputFileHandler3 = open(inputFileName3, "r")
    for currentLine in inputFileHandler3.readlines():
        currentLineElements = currentLine.strip().split(" ")
        word = currentLineElements[0]
        docID = currentLineElements[1]
        position = currentLineElements[2]
        context = currentLineElements[3]
        BM25Score = currentLineElements[4]
        key = (word, docID, BM25Score)
        if key not in termDict:
            termDict[key] = [(position,context)]
        else:
            #print termDict[key]
            termDict[key] = termDict[key] + [(position,context)]
            
    inputFileHandler3.close()
    #print len(termDict)
    
    
    #for key in termDict:
    #    print key,termDict[key]
    #print termDict[("day","55")]
    #print termDict[("day","7")]
    
    
    #Let's construct a query_document_term dict
    return_query_document_term_Dict = {}
    
    noPositionCounter = 0
    for currentElement in query_document_Dict:
        #Copy all the info from the previous dict first.
        if currentElement not in return_query_document_term_Dict:
            return_query_document_term_Dict[currentElement] = query_document_Dict[currentElement]
        else:
            print "Error3"
            print currentElement
            sys.exit(1)
        
        #copy all the term level from the term dict to the return dict.
        (queryContent,docID) = currentElement
        queryContentElements = queryContent.split(" ")
        for queryTerm in queryContentElements:
            eachTermAndIDFlag = False
            for eachTuple in termDict:
                (word,wordDocID,wordBM25) = eachTuple
                if word == queryTerm and wordDocID == docID:
                    
                    eachTermAndIDFlag = True
                    
                    termInfoString = str(termDict[eachTuple]) + " " + wordBM25 + " "
                    
                    return_query_document_term_Dict[currentElement] += termInfoString
                    #query_document_Dict[currentElement] += termInfoString
                    '''
                    if (queryTerm, docID) in termDict:
                        newTurpleString = "(" + queryTerm + "," + docID + "," + str( len(termDict[(queryTerm, docID)]) ) + ","
                        for eachPositionInfo in termDict[(queryTerm, docID)][: len(termDict[(queryTerm, docID)])-1]:
                            newTurpleString += eachPositionInfo + ","
                        newTurpleString += termDict[(queryTerm, docID)][-1]
                        newTurpleString += ")"
                        #print newTurpleString
                        #newTurple = tuple(newTurpleString)
                        
                        if newTurpleString not in return_query_document_term_Dict:
                            return_query_document_term_Dict[newTurpleString] = query_document_Dict[currentElement]
                        
                    else:
                        #print "NO Key:",queryTerm, docID
                        noPositionCounter += 1
                        newTurpleString = "(" + queryTerm + "," + docID + "," + str( 0 ) + ")"
                        #newTurple = tuple(newTurpleString)
                        if newTurpleString not in return_query_document_term_Dict:
                            return_query_document_term_Dict[newTurpleString] = query_document_Dict[currentElement]
                        else:
                            print "Error3"
                            print newTurpleString
                            sys.exit(1)
                    '''
            if not eachTermAndIDFlag:
                #fake position context list and fake BM25 score.
                termInfoString = str([]) + " " + str(0) + " "
                return_query_document_term_Dict[currentElement] += termInfoString
                
                noPositionCounter += 1
                print "No Position and Context (queryTerm, docID) pair in termDict:",queryTerm,docID
                
                
    
    print "Word appeared in the query but No Position and Context info in document Counter:",noPositionCounter
    
    return return_query_document_term_Dict
    
def loadNeededDocuments(inputFileName1):
    inputFileForUseEvaluatedDocumentsInfoFileHandler = open(inputFileName1, "r")
    finalDocumentDict = {}
        
    for currentLine in inputFileForUseEvaluatedDocumentsInfoFileHandler.readlines():
        currentLineElements = currentLine.split("\t")
        WARCTRECID = currentLineElements[3]
        if WARCTRECID not in finalDocumentDict:
            finalDocumentDict[WARCTRECID] = 1
        else:
            finalDocumentDict[WARCTRECID] += 1
    inputFileForUseEvaluatedDocumentsInfoFileHandler.close()
        
    
    return finalDocumentDict

#main program
inputFileName1 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/query_features_2011_adhoc.txt"

dict0 = loadNeededDocuments(inputFileName1)
print "# of documents needed to be collected:",len(dict0)



dict1 = dealWithQueryLevelFeature(inputFileName1)
print len(dict1)
for queryWARCTRECIDTupleElement in dict1:
    print queryWARCTRECIDTupleElement,dict1[queryWARCTRECIDTupleElement]


inputFileName2 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/document_features.txt"
dict2 = dealWithDocumentLevelFeature( dict0, dict1,inputFileName2 )
print len(dict2)
for queryWARCTRECIDDocIDTupleElement in dict2:
    print queryWARCTRECIDDocIDTupleElement,dict2[queryWARCTRECIDDocIDTupleElement]


#inputFileName3 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/term_features.txt"
#dict3 = dealWithTermLevelFeature( dict2, inputFileName3 )
#print len(dict3)
#for queryDocIDTermTupleElement in dict3:
#    print queryDocIDTermTupleElement,dict3[queryDocIDTermTupleElement]


 
    
    
    
